# -*- coding: utf-8 -*-
import scrapy
from dingxiangyuan.DBHelper import redis_conn
import json
from dingxiangyuan.items import DingxiangyuanItemLoader, UserFollowerItem
from dingxiangyuan.DBHelper import get_user_follower_dict


class UserFollowerSpider(scrapy.Spider):
    name = 'user_follower'
    allowed_domains = ['i.dxy.cn']
    # start_urls = ['http://i.dxy.cn/snsapi/friend/following/listAll?JUTE_TOKEN=c1a9caca-2a10-452a-8ec7-8226583e96a7&userId=1&page=1&size=20']

    def start_requests(self):
        JUTE_TOKEN = '9c271aa3-ffbf-4ea3-b021-13b57697be51'
        page_size = 20
        user_infos = redis_conn.smembers('user_follower_infos')
        # data_dict = get_user_follower_dict()
        for user_info in user_infos:
            user_url, user_name, following, follower, user_id = user_info.decode().split(';')
            div, mod = divmod(int(follower), page_size)
            pages = div + 1 if mod > 0 else div
            pages = pages if pages < 10000 else 9999

            div2, mod2 = divmod(int(following), page_size)
            pages2 = div2 + 1 if mod2 > 0 else div2
            pages2 = pages2 if pages2 < 10000 else 9999

            # if data_dict[int(user_id)] < int(follower):
            for page in range(1, pages + 1):
                start_url = f'http://i.dxy.cn/snsapi/friend/follower/listAll?JUTE_TOKEN={JUTE_TOKEN}&userId={user_id}&page={page}&size={page_size}'
                yield scrapy.Request(url=start_url, meta={'user_name': user_name, 'user_id': user_id}, callback=self.parse)

            for page in range(1, pages2 + 1):
                start_url = f'http://i.dxy.cn/snsapi/friend/following/listAll?JUTE_TOKEN={JUTE_TOKEN}&userId={user_id}&page={page}&size={page_size}'
                yield scrapy.Request(url=start_url, meta={'user_name': user_name, 'user_id': user_id}, callback=self.parse_following)

    def parse(self, response):
        user_name = response.meta['user_name']
        user_id = response.meta['user_id']
        data = json.loads(response.text)
        for item in data['items']:
            item_loader = DingxiangyuanItemLoader(item=UserFollowerItem())
            item_loader.add_value('user_id', user_id)
            item_loader.add_value('user_name', user_name)
            item_loader.add_value('fans_id', item['userId'])
            item_loader.add_value('fans_name', item['userName'])
            user_follower_item = item_loader.load_item()
            yield user_follower_item

    def parse_following(self, response):
        user_name = response.meta['user_name']
        user_id = response.meta['user_id']
        data = json.loads(response.text)
        for item in data['items']:
            item_loader = DingxiangyuanItemLoader(item=UserFollowerItem())
            item_loader.add_value('user_id', item['userId'])
            item_loader.add_value('user_name', item['userName'])
            item_loader.add_value('fans_id', user_id)
            item_loader.add_value('fans_name', user_name)
            user_follower_item = item_loader.load_item()
            yield user_follower_item
